{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kR-4eNdK6lYS"
   },
   "source": [
    "Deep Learning\n",
    "=============\n",
    "\n",
    "Assignment 2\n",
    "------------\n",
    "\n",
    "Previously in `1_notmnist.ipynb`, we created a pickle with formatted datasets for training, development and testing on the [notMNIST dataset](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html).\n",
    "\n",
    "The goal of this assignment is to progressively train deeper and more accurate models using TensorFlow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "JLpLa8Jt7Vu4"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from six.moves import cPickle as pickle\n",
    "from six.moves import range"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1HrCK6e17WzV"
   },
   "source": [
    "First reload the data we generated in `1_notmnist.ipynb`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 19456,
     "status": "ok",
     "timestamp": 1449847956073,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "y3-cj1bpmuxc",
    "outputId": "0ddb1607-1fc4-4ddb-de28-6c7ab7fb0c33"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 28, 28) (200000,)\n",
      "Validation set (10000, 28, 28) (10000,)\n",
      "Test set (10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "pickle_file = 'notMNIST.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    save = pickle.load(f)\n",
    "    train_dataset = save['train_dataset']\n",
    "    train_labels = save['train_labels']\n",
    "    valid_dataset = save['valid_dataset']\n",
    "    valid_labels = save['valid_labels']\n",
    "    test_dataset = save['test_dataset']\n",
    "    test_labels = save['test_labels']\n",
    "    del save  # hint to help gc free up memory\n",
    "    print('Training set', train_dataset.shape, train_labels.shape)\n",
    "    print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "    print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L7aHrm6nGDMB"
   },
   "source": [
    "Reformat into a shape that's more adapted to the models we're going to train:\n",
    "- data as a flat matrix,\n",
    "- labels as float 1-hot encodings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 19723,
     "status": "ok",
     "timestamp": 1449847956364,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "IRSyYiIIGIzS",
    "outputId": "2ba0fc75-1487-4ace-a562-cf81cae82793"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 784) (200000, 10)\n",
      "Validation set (10000, 784) (10000, 10)\n",
      "Test set (10000, 784) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "image_size = 28\n",
    "num_labels = 10\n",
    "\n",
    "def reformat(dataset, labels):\n",
    "    dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)\n",
    "    # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]\n",
    "    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n",
    "    return dataset, labels\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200000, 784)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.shape(train_dataset.reshape((-1, 28*28)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  1.], dtype=float32)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(train_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nCLVqyQ5vPPH"
   },
   "source": [
    "We're first going to train a multinomial logistic regression using simple gradient descent.\n",
    "\n",
    "TensorFlow works like this:\n",
    "* First you describe the computation that you want to see performed: what the inputs, the variables, and the operations look like. These get created as nodes over a computation graph. This description is all contained within the block below:\n",
    "\n",
    "      with graph.as_default():\n",
    "          ...\n",
    "\n",
    "* Then you can run the operations on this graph as many times as you want by calling `session.run()`, providing it outputs to fetch from the graph that get returned. This runtime operation is all contained in the block below:\n",
    "\n",
    "      with tf.Session(graph=graph) as session:\n",
    "          ...\n",
    "\n",
    "Let's load all the data into TensorFlow and build the computation graph corresponding to our training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "Nfv39qvtvOl_"
   },
   "outputs": [],
   "source": [
    "# With gradient descent training, even this much data is prohibitive.\n",
    "# Subset the training data for faster turnaround.\n",
    "train_subset = 10000\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data.\n",
    "    # Load the training, validation and test data into constants that are\n",
    "    # attached to the graph.\n",
    "    tf_train_dataset = tf.constant(train_dataset[:train_subset, :])\n",
    "    tf_train_labels = tf.constant(train_labels[:train_subset])\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "    # Variables.\n",
    "    # These are the parameters that we are going to be training. The weight\n",
    "    # matrix will be initialized using random values following a (truncated)\n",
    "    # normal distribution. The biases get initialized to zero.\n",
    "    weights = tf.Variable(\n",
    "    tf.truncated_normal([image_size * image_size, num_labels]))\n",
    "    biases = tf.Variable(tf.zeros([num_labels]))\n",
    "  \n",
    "    # Training computation.\n",
    "    # We multiply the inputs with the weight matrix, and add biases. We compute\n",
    "    # the softmax and cross-entropy (it's one operation in TensorFlow, because\n",
    "    # it's very common, and it can be optimized). We take the average of this\n",
    "    # cross-entropy across all training examples: that's our loss.\n",
    "    logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "  \n",
    "    # Optimizer.\n",
    "    # We are going to find the minimum of this loss using gradient descent.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "  \n",
    "    # Predictions for the training, validation, and test data.\n",
    "    # These are not part of training, but merely here so that we can report\n",
    "    # accuracy figures as we train.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(tf_valid_dataset, weights) + biases)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KQcL4uqISHjP"
   },
   "source": [
    "Let's run this computation and iterate:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "accur_list = []\n",
    "cost_list = []\n",
    "step_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 9
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 57454,
     "status": "ok",
     "timestamp": 1449847994134,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "z2cjdenH869W",
    "outputId": "4c037ba1-b526-4d8e-e632-91e2a0333267"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Loss at step 0: 15.014024\n",
      "Training accuracy: 17.2%\n",
      "Validation accuracy: 19.5%\n",
      "Test accuracy: 20.8%\n",
      "Loss at step 10: 5.785231\n",
      "Training accuracy: 40.8%\n",
      "Validation accuracy: 42.8%\n",
      "Test accuracy: 48.2%\n",
      "Loss at step 20: 4.207510\n",
      "Training accuracy: 54.3%\n",
      "Validation accuracy: 55.2%\n",
      "Test accuracy: 62.1%\n",
      "Loss at step 30: 3.621684\n",
      "Training accuracy: 60.7%\n",
      "Validation accuracy: 60.6%\n",
      "Test accuracy: 67.8%\n",
      "Loss at step 40: 3.283602\n",
      "Training accuracy: 63.7%\n",
      "Validation accuracy: 63.7%\n",
      "Test accuracy: 71.0%\n",
      "Loss at step 50: 3.051245\n",
      "Training accuracy: 65.9%\n",
      "Validation accuracy: 65.7%\n",
      "Test accuracy: 73.3%\n",
      "Loss at step 60: 2.877312\n",
      "Training accuracy: 67.5%\n",
      "Validation accuracy: 67.0%\n",
      "Test accuracy: 74.7%\n",
      "Loss at step 70: 2.739522\n",
      "Training accuracy: 68.6%\n",
      "Validation accuracy: 67.9%\n",
      "Test accuracy: 75.7%\n",
      "Loss at step 80: 2.625506\n",
      "Training accuracy: 69.6%\n",
      "Validation accuracy: 68.6%\n",
      "Test accuracy: 76.5%\n",
      "Loss at step 90: 2.528108\n",
      "Training accuracy: 70.3%\n",
      "Validation accuracy: 69.2%\n",
      "Test accuracy: 77.1%\n",
      "Loss at step 100: 2.442977\n",
      "Training accuracy: 70.9%\n",
      "Validation accuracy: 69.8%\n",
      "Test accuracy: 77.7%\n",
      "Loss at step 110: 2.367375\n",
      "Training accuracy: 71.3%\n",
      "Validation accuracy: 70.3%\n",
      "Test accuracy: 78.1%\n",
      "Loss at step 120: 2.299448\n",
      "Training accuracy: 71.7%\n",
      "Validation accuracy: 70.6%\n",
      "Test accuracy: 78.5%\n",
      "Loss at step 130: 2.237887\n",
      "Training accuracy: 71.9%\n",
      "Validation accuracy: 71.0%\n",
      "Test accuracy: 78.9%\n",
      "Loss at step 140: 2.181731\n",
      "Training accuracy: 72.2%\n",
      "Validation accuracy: 71.2%\n",
      "Test accuracy: 79.2%\n",
      "Loss at step 150: 2.130227\n",
      "Training accuracy: 72.5%\n",
      "Validation accuracy: 71.6%\n",
      "Test accuracy: 79.4%\n",
      "Loss at step 160: 2.082773\n",
      "Training accuracy: 72.8%\n",
      "Validation accuracy: 72.0%\n",
      "Test accuracy: 79.5%\n",
      "Loss at step 170: 2.038877\n",
      "Training accuracy: 73.0%\n",
      "Validation accuracy: 72.1%\n",
      "Test accuracy: 79.7%\n",
      "Loss at step 180: 1.998118\n",
      "Training accuracy: 73.3%\n",
      "Validation accuracy: 72.3%\n",
      "Test accuracy: 79.9%\n",
      "Loss at step 190: 1.960131\n",
      "Training accuracy: 73.5%\n",
      "Validation accuracy: 72.5%\n",
      "Test accuracy: 80.1%\n",
      "Loss at step 200: 1.924603\n",
      "Training accuracy: 73.7%\n",
      "Validation accuracy: 72.6%\n",
      "Test accuracy: 80.3%\n",
      "Loss at step 210: 1.891265\n",
      "Training accuracy: 73.9%\n",
      "Validation accuracy: 72.8%\n",
      "Test accuracy: 80.4%\n",
      "Loss at step 220: 1.859890\n",
      "Training accuracy: 74.0%\n",
      "Validation accuracy: 72.9%\n",
      "Test accuracy: 80.5%\n",
      "Loss at step 230: 1.830279\n",
      "Training accuracy: 74.2%\n",
      "Validation accuracy: 73.0%\n",
      "Test accuracy: 80.6%\n",
      "Loss at step 240: 1.802266\n",
      "Training accuracy: 74.3%\n",
      "Validation accuracy: 73.1%\n",
      "Test accuracy: 80.7%\n",
      "Loss at step 250: 1.775700\n",
      "Training accuracy: 74.5%\n",
      "Validation accuracy: 73.2%\n",
      "Test accuracy: 80.8%\n",
      "Loss at step 260: 1.750453\n",
      "Training accuracy: 74.5%\n",
      "Validation accuracy: 73.3%\n",
      "Test accuracy: 80.9%\n",
      "Loss at step 270: 1.726411\n",
      "Training accuracy: 74.6%\n",
      "Validation accuracy: 73.4%\n",
      "Test accuracy: 80.9%\n",
      "Loss at step 280: 1.703474\n",
      "Training accuracy: 74.7%\n",
      "Validation accuracy: 73.4%\n",
      "Test accuracy: 81.0%\n",
      "Loss at step 290: 1.681553\n",
      "Training accuracy: 74.8%\n",
      "Validation accuracy: 73.4%\n",
      "Test accuracy: 81.1%\n",
      "Loss at step 300: 1.660568\n",
      "Training accuracy: 74.9%\n",
      "Validation accuracy: 73.6%\n",
      "Test accuracy: 81.2%\n",
      "Loss at step 310: 1.640448\n",
      "Training accuracy: 75.0%\n",
      "Validation accuracy: 73.6%\n",
      "Test accuracy: 81.2%\n",
      "Loss at step 320: 1.621129\n",
      "Training accuracy: 75.1%\n",
      "Validation accuracy: 73.7%\n",
      "Test accuracy: 81.2%\n",
      "Loss at step 330: 1.602553\n",
      "Training accuracy: 75.3%\n",
      "Validation accuracy: 73.7%\n",
      "Test accuracy: 81.3%\n",
      "Loss at step 340: 1.584666\n",
      "Training accuracy: 75.4%\n",
      "Validation accuracy: 73.8%\n",
      "Test accuracy: 81.3%\n",
      "Loss at step 350: 1.567423\n",
      "Training accuracy: 75.5%\n",
      "Validation accuracy: 73.9%\n",
      "Test accuracy: 81.3%\n",
      "Loss at step 360: 1.550779\n",
      "Training accuracy: 75.6%\n",
      "Validation accuracy: 73.9%\n",
      "Test accuracy: 81.3%\n",
      "Loss at step 370: 1.534697\n",
      "Training accuracy: 75.7%\n",
      "Validation accuracy: 73.9%\n",
      "Test accuracy: 81.4%\n",
      "Loss at step 380: 1.519141\n",
      "Training accuracy: 75.8%\n",
      "Validation accuracy: 74.0%\n",
      "Test accuracy: 81.4%\n",
      "Loss at step 390: 1.504077\n",
      "Training accuracy: 75.8%\n",
      "Validation accuracy: 74.0%\n",
      "Test accuracy: 81.4%\n",
      "Loss at step 400: 1.489481\n",
      "Training accuracy: 75.9%\n",
      "Validation accuracy: 74.0%\n",
      "Test accuracy: 81.5%\n",
      "Loss at step 410: 1.475324\n",
      "Training accuracy: 76.0%\n",
      "Validation accuracy: 74.0%\n",
      "Test accuracy: 81.5%\n",
      "Loss at step 420: 1.461582\n",
      "Training accuracy: 76.1%\n",
      "Validation accuracy: 74.0%\n",
      "Test accuracy: 81.6%\n",
      "Loss at step 430: 1.448232\n",
      "Training accuracy: 76.2%\n",
      "Validation accuracy: 74.1%\n",
      "Test accuracy: 81.6%\n",
      "Loss at step 440: 1.435256\n",
      "Training accuracy: 76.1%\n",
      "Validation accuracy: 74.1%\n",
      "Test accuracy: 81.6%\n",
      "Loss at step 450: 1.422634\n",
      "Training accuracy: 76.3%\n",
      "Validation accuracy: 74.1%\n",
      "Test accuracy: 81.6%\n",
      "Loss at step 460: 1.410349\n",
      "Training accuracy: 76.3%\n",
      "Validation accuracy: 74.1%\n",
      "Test accuracy: 81.7%\n",
      "Loss at step 470: 1.398384\n",
      "Training accuracy: 76.5%\n",
      "Validation accuracy: 74.1%\n",
      "Test accuracy: 81.7%\n",
      "Loss at step 480: 1.386726\n",
      "Training accuracy: 76.5%\n",
      "Validation accuracy: 74.2%\n",
      "Test accuracy: 81.7%\n",
      "Loss at step 490: 1.375358\n",
      "Training accuracy: 76.6%\n",
      "Validation accuracy: 74.3%\n",
      "Test accuracy: 81.7%\n",
      "Loss at step 500: 1.364270\n",
      "Training accuracy: 76.7%\n",
      "Validation accuracy: 74.3%\n",
      "Test accuracy: 81.8%\n",
      "Loss at step 510: 1.353448\n",
      "Training accuracy: 76.7%\n",
      "Validation accuracy: 74.3%\n",
      "Test accuracy: 81.8%\n",
      "Loss at step 520: 1.342881\n",
      "Training accuracy: 76.8%\n",
      "Validation accuracy: 74.3%\n",
      "Test accuracy: 81.8%\n",
      "Loss at step 530: 1.332558\n",
      "Training accuracy: 76.9%\n",
      "Validation accuracy: 74.3%\n",
      "Test accuracy: 81.8%\n",
      "Loss at step 540: 1.322469\n",
      "Training accuracy: 76.9%\n",
      "Validation accuracy: 74.3%\n",
      "Test accuracy: 81.8%\n",
      "Loss at step 550: 1.312604\n",
      "Training accuracy: 76.9%\n",
      "Validation accuracy: 74.4%\n",
      "Test accuracy: 81.9%\n",
      "Loss at step 560: 1.302956\n",
      "Training accuracy: 77.0%\n",
      "Validation accuracy: 74.5%\n",
      "Test accuracy: 81.9%\n",
      "Loss at step 570: 1.293516\n",
      "Training accuracy: 77.0%\n",
      "Validation accuracy: 74.5%\n",
      "Test accuracy: 82.0%\n",
      "Loss at step 580: 1.284275\n",
      "Training accuracy: 77.1%\n",
      "Validation accuracy: 74.5%\n",
      "Test accuracy: 82.0%\n",
      "Loss at step 590: 1.275225\n",
      "Training accuracy: 77.1%\n",
      "Validation accuracy: 74.5%\n",
      "Test accuracy: 82.0%\n",
      "Loss at step 600: 1.266361\n",
      "Training accuracy: 77.2%\n",
      "Validation accuracy: 74.6%\n",
      "Test accuracy: 82.0%\n",
      "Loss at step 610: 1.257675\n",
      "Training accuracy: 77.2%\n",
      "Validation accuracy: 74.6%\n",
      "Test accuracy: 82.0%\n",
      "Loss at step 620: 1.249164\n",
      "Training accuracy: 77.2%\n",
      "Validation accuracy: 74.6%\n",
      "Test accuracy: 82.0%\n",
      "Loss at step 630: 1.240817\n",
      "Training accuracy: 77.2%\n",
      "Validation accuracy: 74.6%\n",
      "Test accuracy: 82.1%\n",
      "Loss at step 640: 1.232631\n",
      "Training accuracy: 77.3%\n",
      "Validation accuracy: 74.6%\n",
      "Test accuracy: 82.2%\n",
      "Loss at step 650: 1.224601\n",
      "Training accuracy: 77.3%\n",
      "Validation accuracy: 74.7%\n",
      "Test accuracy: 82.2%\n",
      "Loss at step 660: 1.216719\n",
      "Training accuracy: 77.4%\n",
      "Validation accuracy: 74.7%\n",
      "Test accuracy: 82.2%\n",
      "Loss at step 670: 1.208985\n",
      "Training accuracy: 77.4%\n",
      "Validation accuracy: 74.7%\n",
      "Test accuracy: 82.1%\n",
      "Loss at step 680: 1.201391\n",
      "Training accuracy: 77.5%\n",
      "Validation accuracy: 74.8%\n",
      "Test accuracy: 82.1%\n",
      "Loss at step 690: 1.193932\n",
      "Training accuracy: 77.6%\n",
      "Validation accuracy: 74.8%\n",
      "Test accuracy: 82.2%\n",
      "Loss at step 700: 1.186607\n",
      "Training accuracy: 77.6%\n",
      "Validation accuracy: 74.8%\n",
      "Test accuracy: 82.2%\n",
      "Loss at step 710: 1.179410\n",
      "Training accuracy: 77.7%\n",
      "Validation accuracy: 74.8%\n",
      "Test accuracy: 82.2%\n",
      "Loss at step 720: 1.172336\n",
      "Training accuracy: 77.8%\n",
      "Validation accuracy: 74.9%\n",
      "Test accuracy: 82.2%\n",
      "Loss at step 730: 1.165383\n",
      "Training accuracy: 77.8%\n",
      "Validation accuracy: 75.0%\n",
      "Test accuracy: 82.3%\n",
      "Loss at step 740: 1.158550\n",
      "Training accuracy: 77.8%\n",
      "Validation accuracy: 74.9%\n",
      "Test accuracy: 82.3%\n",
      "Loss at step 750: 1.151828\n",
      "Training accuracy: 77.9%\n",
      "Validation accuracy: 74.9%\n",
      "Test accuracy: 82.4%\n",
      "Loss at step 760: 1.145218\n",
      "Training accuracy: 77.9%\n",
      "Validation accuracy: 74.9%\n",
      "Test accuracy: 82.4%\n",
      "Loss at step 770: 1.138715\n",
      "Training accuracy: 78.0%\n",
      "Validation accuracy: 74.9%\n",
      "Test accuracy: 82.4%\n",
      "Loss at step 780: 1.132317\n",
      "Training accuracy: 78.0%\n",
      "Validation accuracy: 74.8%\n",
      "Test accuracy: 82.4%\n",
      "Loss at step 790: 1.126021\n",
      "Training accuracy: 78.0%\n",
      "Validation accuracy: 74.9%\n",
      "Test accuracy: 82.4%\n",
      "Loss at step 800: 1.119824\n",
      "Training accuracy: 78.1%\n",
      "Validation accuracy: 74.9%\n",
      "Test accuracy: 82.4%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 801\n",
    "\n",
    "def accuracy(predictions, labels):\n",
    "    return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    # This is a one-time operation which ensures the parameters get initialized as\n",
    "    # we described in the graph: random weights for the matrix, zeros for the\n",
    "    # biases. \n",
    "    tf.initialize_all_variables().run()\n",
    "    print('Initialized')\n",
    "    for step in range(num_steps):\n",
    "        # Run the computations. We tell .run() that we want to run the optimizer,\n",
    "        # and get the loss value and the training predictions returned as numpy\n",
    "        # arrays.\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction])\n",
    "        if (step % 10 == 0):\n",
    "            cost_list.append(l)\n",
    "            step_list.append(step)\n",
    "            print('Loss at step %d: %f' % (step, l))\n",
    "            print('Training accuracy: %.1f%%' % accuracy(\n",
    "            predictions, train_labels[:train_subset, :]))\n",
    "            # Calling .eval() on valid_prediction is basically like calling run(), but\n",
    "            # just to get that one numpy array. Note that it recomputes all its graph\n",
    "            # dependencies.\n",
    "            print('Validation accuracy: %.1f%%' % accuracy(valid_prediction.eval(), valid_labels))\n",
    "            T_accuracy = accuracy(test_prediction.eval(), test_labels)\n",
    "            print('Test accuracy: %.1f%%' % T_accuracy)\n",
    "            accur_list.append(T_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEZCAYAAACU3p4jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHPhJREFUeJzt3XucXGWd5/HPNwnhntABSZRuIpdBgiKIclnBpRwY9KUI\njINRAblkCY7ogu6CAXYmCczqAoOyjugqgWDEAMnAAHFWJQRSw4hy0wCRcBGQ0GFIKyQEBEEgv/nj\nPJ0UnaqkqrtPV1Wf7/v1Oq8+9/PUk86vnv6d5zxHEYGZmQ1/I5pdADMzGxoO+GZmBeGAb2ZWEA74\nZmYF4YBvZlYQDvhmZgXhgG9mVhAO+DYkJJUlrZK0WbPL0sok7STpekl/kLRa0oOSTkzbJkpaK8n/\nb61f/ItjuZM0ETgEWAscNcTXHjmU1xsEVwPLgS5ge+BzQE/aJiDST7OGOeDbUDgR+CXwA+Dkyg2S\ntpD0DUlPpRbtHZI2T9sOkXRnWr+8oqW7WNKUinOcJOnfK5bXSjpd0mPAY2nd/5X0tKQ1ku6VdEjF\n/iMknSfpcUkvpu07SbpM0iV9ynuzpDP7fkBJ35X0j33W3STpy2l+mqQV6fwPS/pwjbraH5gTEa9G\nxNqIeCAibknb/i39fCGd58B07imSlkl6XtJPJe3cpy7+u6QnJP1e0sUV23ZLf3m9kLZdW6NMNlxE\nhCdPuU7Ab4HPA/sBfwbeVrHtO8DtwASylutBwGbAzsCLwGRgJNABvDcdsxiYUnGOk4A7KpbXArcA\nY4HN07rjgO3IGjlfAZ4FRqdtZwMPALun5b3T9fYHVlScd3vgj8AOVT7jh4DlFcvbAS8D44E9gKeB\n8WnbzsAuNepqIfBz4NNAV59tE4E3AVWsO5rsS22P9NnOA+7sUxe3pbroBB7trTvgGuDcND8a+GCz\nf1c85Ts1vQCehvdElsp5DehIy8uAM9O8gFeA91Q57hzghhrnrCfgH7qJcq0C9k7zjwBH1tjvIeCw\nNP9F4F83cs6ngEPS/KnAojS/G7ASOAwYtYlyjQW+DiwFXgd+DXwgbesN+CMq9v8JcErF8oj0RdNV\nURd/VbH9C8CtaX4O8D1gp2b/nngamskpHcvbicDCiFidlq8lC9AAOwCbA09WOa4LeGIA111RuSDp\nrJT2WC1pNTAmXb/3WtXKAPBD4IQ0fwJZjr2WecBn0/xxwFyAiHgC+DIwE+iRdI2kt1c7QUSsiYjz\nImJvsr8OHgBu3Mg1JwLfSjfEVwHPk+X5d6rYp7IulgPvSPNfJfuCuEfSUkmnbOQ6Ngw44FtuJG1B\nlpI5VNKzkp4lC3z7SNobeA54lawF3Fc3sHuNU78MbFWxPKHKPuuGgU35+rOBYyOiIyI6yNJFvTc/\nu2uUAeBHwNGS3gvsCdxUYz/IvsyOTTn0A4Eb1hUm4rqI+BBZgAa4cCPn6T1mFXAJ8A5JHZWfqcLT\nwOcjYlyaOiJim4i4q2Kfror5nYH/SOfviYjTImIn4G+B70radVPlsvblgG95+mvgDWASsE+aJpHl\nqE+MiACuAr4p6e3p5ulBqevmXOAwScdKGilpnKR90nnvBz4paUtJuwP/bRPl2JYsPfK8pNGSpqd1\nva4A/iGdC0l7pwBLRDwD3EfWsr8hIl6rdZGIuJ+shX0F8LOIeDGdbw9JH5Y0muwexp/IUi0bkHSh\npHenz7wtcDrwePoL6Q/puMovp+8D50naKx0/VtKxfU57tqTtJHUBZwLXpX2PldT7l8AL6dxVy2XD\ngwO+5elEYHZEPBMRv++dgMuA41N/8rPI8tX3kgXLC8ly1N3Ax9L2VcAS4L3pvJeSBfCVZF8YP+pz\n3b4t4VvS9BjwO7L7Bt0V278JzAcWSlpDFrC3rNg+B3gPWXpnU64hy9XPrVi3efpcfyBrXb8NOLfG\n8VuRpXBWA4+Ttc6PAoiIPwFfA+5MKZwDIuKmdO7rJL0APAh8tM85bwZ+RXY/4MfA7LR+f+BuSS+S\n/eVyRkQ8VcdntDalrJGV4wWyLmynpsVZEfFPqfU0j+zP26eAyRGxJteCmPWTpA8BV0fEO5tdlkZJ\nWkvW+6jWPQorkFxb+JLeTfbn9geAfYEjJe1G1gNjUUS8i6xLXq3WjllTpfTSmcCsZpfFbKDyTulM\nAu6OiNci4k3gDuCTZH+izkn7zAGOybkcZg2TtCdZamU88K0mF6e//A5TWyfXlE76D3MT8F/I+mIv\nIrsBdkJEjKvYb1XlspmZDb5ReZ48Ih6RdBFwK9kTikvIHhzZYNc8y2FmZjkHfICIuIqsJwWSvkbW\nO6JH0viI6JE0Afh9tWMl+YvAzKwfImKDQfZy75Yp6W3p585k/bKvARawfhCtk8i6jVXV7EeRZ8yY\n0fQytMrkunBduC7aoy5qyb2FD9wgaRxZv+nTI+LFlOaZn0Y8XE72NKaZmeVoKFI6/7XKulXA4Xlf\n28zM1vOTtptQKpWaXYSW4bpYz3WxnutivVavi9yftB0ISdHK5TMza0WSiGbctDUzs9bggG9mVhAO\n+GZmBeGAb2ZWEA74ZmYF4YBvZlYQDvhmZgXhgG9mVhAO+GZmBeGAb2ZWEA74ZmYF4YBvZlYQDvhm\nZgXhgG9mVhAO+GbWVLNmzaaray9mzZr9lvl6j9nYOerdbzDOUW/Zm6rZ717cxHsZw8wad/nlV0Zn\n56S4/PIra863yn6dnZMCTozOzklvmd/Y+fruV+sc9e43GOfou18zpdi5YUyttrJVJgd8axWtHjT7\nbhvs4JXnfv35HHnU2WD/+zRT0wI+8BXgN8CDwFxgNNABLAQeBW4BxtY4NudqsXbQCgG11YNmvS3j\nVvyy6s+/vW1cUwI+8A7gSWB0Wp4HnARcBHw1rZsGXFjj+HxrxYZUO7dWWz1oOhhapWYG/OWpRT8K\nWAAcDjwCjE/7TAAeqXF8ztVi/TVUreT+XmsoW6FmraaZKZ0zgJeAHuDqtG51n31W1Tg2xyqxTRns\nVrdbq2ZDo1kt/O2A24BxwEjgX4Dj+wZ44Pkax+daKbZhK7beFrlbyWatq1bAV7YtH5KOBT4SEVPT\n8ueAg4C/BEoR0SNpArA4IiZVOT5mzJixbrlUKlEqlXIr73A2a9ZsLrjgEqZPPwtg3fwFF1zCihX7\n09l5L93dy+jq2mvdcu/2vsdMnTqlmR/FzPool8uUy+V1y+effz4RoQ12rPYtMFgTcACwFNgCEPAD\n4ItkN22npX180zYH9aZg3CI3G35oYg5/BvAwWbfMOcBmZCmeRWTdMhcC29U4NudqGV5qpWP6bnNQ\nNxveagX8XFM6AyUpWrl8raAyVVOZnqlMxzgFY1YskqqmdBzw20xlgJ86dUrNnLuDvFlxOeC3sVqt\n+O7uZRt8AZiZOeC3GadqzKy/HPDbjFM1ZtZftQK+x8NvEX3H0p4+/ax1wX7q1Cl0dy9zsDezAXEL\nv0VUtui7u5c1uzhm1sbcwm9xlS16M7M8uIXfRO5hY2Z58E3bFuQ0jpnlwSmdFlF5c9ZpHDMbSm7h\nDzG36s0sb27htwi36s2sWdzCNzMbZtzCb5K+D1SZmTWLW/g5c87ezIaaW/hN4py9mbUKt/DNzIYZ\nt/DNzArOAd/MrCByDfiS9pC0RNKv0881ks6Q1CFpoaRHJd0iaWye5Rhq7pljZq1oyHL4kkYAK4AD\ngS8Bz0fExZKmAR0RcU6VY9oyh++eOWbWTK2Qwz8ceCIiuoGjgTlp/RzgmCEsR+7cM8fMWtFQtvCv\nBO6LiP8naXVEdFRsWxUR46oc05YtfDOzZqrVwh81RBffDDgKmJZW9Y3iNaP6zJkz182XSiVKpdIg\nl87MrL2Vy2XK5fIm9xuSFr6ko4DTI+KjaflhoBQRPZImAIsjYlKV49zCNzNrULNz+J8Frq1YXgCc\nnOZPAm4eonKYmRVW7i18SVsBy4FdI+KltG4cMB/oStsmR8QLVY51C9/MrEF+xaGZWUE0O6VjZmZN\n5oA/CPxkrZm1A6d0BoGfrDWzVuKUTo78ZK2ZtQO38M3Mhhm38M3MCs4B38ysIBzwzcwKwgHfzKwg\nHPDNzArCAd/MrCAc8M3MCsIB38ysIBzwzcwKwgHfzKwgHPDNzArCAb+fPCSymbUbD57WTx4S2cxa\nVb8HT5N0t6TPSxqTT9Hak4dENrN2s8kWvqQ9gVOATwG/AK6KiNvqvoA0FrgCeA+wFpgCPAbMAyYC\nT5G9xHxNlWNbtoVvZtaqBvwSc0kjgaOAy4A/A7OBb0fEC5s47gfAv0XEVZJGAVsD5wHPR8TFkqYB\nHRFxTpVjHfDNzBo0oIAvaS+yVv4ngNuBucAhwKcjYr+NHDcGWBIRu/VZ/whwaET0SJoAlCNizyrH\nO+CbmTWoVsAfVceB9wCvkLXop0fEn9KmOyUdvInDdwGek3QVsA9wH/BlYHxE9ABExEpJO9b/UczM\nrD82GfCBEyLisWobIuKoOs6/H/DFiLhP0qXAOUDfZnvNZvzMmTPXzZdKJUqlUh1FNjMrjnK5TLlc\n3uR+9dy0/QfgG725ekkdwJcjYsYmTy6NB34ZEbum5UPIAv5uQKkipbM4IiZVOd4pHTOzBg3knbZH\nVt6YjYjVZLn8TUppm25Je6RVhwEPAQuAk9O6k4Cb6zmfmZn1Xz0pnZGSRkfEnwEkbQGMbuAaZwBz\nJW0GPEl283ckMF/SFGA5MLmxYpuZWaPqSemcB3yE7KYtZP3ofxYR/yfnsjmlY2bWDwPtlvkJsnQM\nwK0R8f8HuXy1ruuAb2bWoAE/eNUMDvhmZo0byFg6+0u6S9IaSa9Kek3Si/kU08zM8lJPL53vkvWk\neRLYFvgS8E95FsrMzAZfPQF/REQ8CoyKiNcjYhbw8ZzLZWZmg6yebpkvSxoNPCDp68CzZN0qzcys\njdTTwj857fcl4E3gL4BjcyyTmZnlYKO9dNKQyFdFxIlDV6S3XN+9dMzMGtSvXjoR8Sawa3pK1szM\n2lg9OfwngH+XdDPwcu/KiHBPHTOzNlJPwH86TVulyczM2pCftDUzG2YG8sarW6nygpKIOGKQymZm\nZkOgnpTO31XMbwH8DfBaPsUxM7O89CulI+nuiDgwh/L0vY5TOmZmDRpISmdMxeII4P1AxyCWzczM\nhkA9KZ2HyHL4At4AfgdMzbNQZmY2+NxLx8xsmBnIePh/K2m7iuUOSacNdgHNzCxf9bzT9v6I2LfP\nuiUR8b66LiA9BawB1gKvR8QBkjqAecBE4ClgckSsqXKsW/hmZg3qdwufPkMhSxoBNDK2zlqgFBHv\ni4gD0rpzgEUR8S7gduDcBs5nZmb9UE/Av1XStZIOlXQoMBdY1MA1VOU6RwNz0vwc4JgGzmdmZv1Q\nT0pnJPAF4PC06lbg+xHxRl0XkJ4EXiAbS//7EXGFpNUR0VGxz6qIGFflWKd0zMwa1O9++GTpm+9G\nxGXpRCOA0WRdNOtxcEQ8K+ltwEJJj7LhUA01o/rMmTPXzZdKJUqlUp2XNTMrhnK5TLlc3uR+9bTw\nfwkcEREvpeVtgVsi4oONFkrSDOCPwKlkef0eSROAxRExqcr+buGbmTVoIDdtt+wN9gBpvq5hkiVt\nJWmbNL81cASwFFhA9upEgJOAm+s5n5mZ9V89KZ1XJO0TEQ8ASNoXeLXO848HbpQU6VpzI2KhpPuA\n+ZKmAMuByf0ou5mZNaCelM6BwLVkgVlAF3BcRNyde+FaLKUza9ZsLrjgEqZPP4upU6c0uzhmZlXV\nSunUNbSCpM2B3hz7MuDN9L7bXLVawO/q2osVK/ans/NeuruXNbs4ZmZVDSSHT0S8FhH3A2OBbwPP\nDHL52sL06WfR2Xkv06ef1eyimJk1rJ6UzgeA48hefLIDcAZwc0Q8l3vhWqyFb2bWDhpO6Ui6APg0\nsJIsh38DcE9E7JJnQfuUwQHfzKxB/Xnw6otkY+FfCvwkIv6cetuYmVkb2lgOfwJwMfAp4ElJVwFb\npidtzcyszdTbS2dL4Cjgs8CBwK0RcWLOZXNKx8ysHwbULbPPibYDPhkRswercBu5lgO+mVmDBi3g\nDyUHfDOzxg2oH76ZmbW/et5pu0FPnmrrzMystdXTwr+nznVmZtbCarbUJe0IvJ2sK+beZAOnAYyh\nzuGRzcysdWwsNfNxYArQCXyH9QH/JeDvcy6XmZkNsnrG0pkcEfOHqDx9r+1eOmZmDRpIL50dJY1J\nJ/mepHskHTboJTQzs1zVE/BPi4gXJR1BltOfSjbkgpmZtZF6An5vTuVjwA/Tqw7df9/MrM3UE7gf\nkPQT4Ejgp+ml5E6sm5m1mXpu2o4E3g88HhGrJO0AdEXEkrovko2weR+wIiKOktQBzAMmAk8BkyNi\nTZXjfNPWzKxB/b5pm95duyvwhbRqy3qO6+NMsnfh9joHWBQR7wJuB85t8HxmZtageoZWuAz4MHBC\nWvUy8L16LyCpkyz/f0XF6qOBOWl+DnBMveczM7P+qael/sGI+DzwKkBErAJGN3CNS4GzeWvef3xE\n9KTzrQR2bOB8ZmbWD/UMgvZ6ysEHgKTtgbX1nFzSx4GeiLhfUmkju9ZM1M+cOXPdfKlUolTa2GnM\nzIqnXC5TLpc3ud/GXmI+KiLekHQi8NfAB4DZwGTg/Ii4bpMnl75Olgp6gyz3vy1wYzpXKSJ6JE0A\nFkfEpCrH+6atmVmDGn4BiqRfR8R+af7dwOFk4+ksiojf9KMAhwL/M/XSuRh4PiIukjQN6IiIc6oc\n44BvZtagWgF/YymddTtHxEPAQ4NYnguB+ZKmAMvJ/mowM7McbayFvwL4Zq0DI6LmtsHiFr6ZWeP6\n08IfCWxDRUvfzMzaV105/GZxC9/MrHH9edLWLXszs2FkYy38cekhq6ZxC9/MrHENd8tsBQ74ZmaN\nG8gbr8zMbBhwwDczKwgHfDOzgnDANzMrCAd8M7OCcMA3MysIB3wzs4JwwDczKwgHfDOzgnDANzMr\nCAd8M7OCcMA3MysIB3wzs4JwwDczK4hcA76kzSXdLWmJpKWSZqT1HZIWSnpU0i2SxuZZDjMzG4Lx\n8CVtFRGvSBoJ3AmcAfwN8HxEXCxpGtAREedUOdbj4ZuZNahp4+FHxCtpdnOyl6YHcDQwJ62fAxyT\ndznMzIou94AvaYSkJcBK4NaIuBcYHxE9ABGxEtgx73KYmRXdqLwvEBFrgfdJGgPcKOndZK38t+xW\n6/iZM2eumy+VSpRKpRxKaWbWvsrlMuVyeZP7Dek7bSX9PfAKcCpQiogeSROAxRExqcr+zuGbmTWo\nKTl8STv09sCRtCXwV8DDwALg5LTbScDNeZbDzMxybuFL2pvspuyINM2LiK9JGgfMB7qA5cDkiHih\nyvFu4ZuZNahWC39IUzqNcsA3M2tc07plmplZa3DANzMrCAd8M7OCcMA3MysIB3wzs4JwwDczKwgH\nfDOzgnDANzMrCAd8M7OCcMA3MysIB3wzs4JwwDczKwgHfDOzgnDANzMrCAd8M7OCcMA3MysIB3wz\ns4JwwDczK4i8X2LeKel2SQ9JWirpjLS+Q9JCSY9KuqX3RedmZpafvF9iPgGYEBH3S9oG+BVwNHAK\n8HxEXCxpGtAREedUOd7vtDUza1BT3mkbESsj4v40/0fgYaCTLOjPSbvNAY7Jsxz9NWvWbLq69mLW\nrNnNLoqZ2YDl2sJ/y4WkdwJl4D1Ad0R0VGxbFRHjqhzT1BZ+V9derFixP52d99Ldvaxp5TAza0RT\nWvgVF98GuB44M7X0+0bxlszbTJ9+Fp2d9zJ9+lnNLoqZ2YDl3sKXNAr4V+CnEfGttO5hoBQRPSnP\nvzgiJlU5NmbMmLFuuVQqUSqVci2vmVm7KZfLlMvldcvnn39+1Rb+UAT8HwLPRcT/qFh3EbAqIi7y\nTVszs8FVK6WTdy+dg4E7gKVkaZsAzgPuAeYDXcByYHJEvFDleAd8M7MGNSXgD5QDvplZ45p609bM\nzJrPAd/MrCAc8M3MCsIB38ysIBzwzcwKwgHfzKwgHPDNzArCAd/MrCAc8M3MCsIB38ysIBzwzcwK\nwgHfzKwgHPDNzArCAd/MrCAc8M3MCsIB38ysIBzwzcwKwgHfzKwgHPDNzAoi14Av6UpJPZIerFjX\nIWmhpEcl3SJpbJ5lMDOzTN4t/KuAj/RZdw6wKCLeBdwOnJtzGQakXC43uwgtw3WxnutiPdfFeq1e\nF7kG/Ij4ObC6z+qjgTlpfg5wTJ5lGKhW/wccSq6L9VwX67ku1mv1umhGDn/HiOgBiIiVwI5NKIOZ\nWeG0wk3baHYBzMyKQBH5xltJE4EfR8R70/LDQCkieiRNABZHxKQax/rLwMysHyJCfdeNGoLrKk29\nFgAnAxcBJwE31zqwWoHNzKx/cm3hS7oGKAHbAz3ADOAm4J+BLmA5MDkiXsitEGZmBgxBSsfMzFpD\nK9y0bVmSPirpEUmPSZrW7PLkTVKnpNslPSRpqaQz0vqaD8tJOlfSbyU9LOmI5pV+8EkaIenXkhak\n5aLWw1hJ/5w+20OSDixwXXxF0m8kPShprqTRbVUXEeGpykT2Zfg4MBHYDLgf2LPZ5cr5M08A9k3z\n2wCPAnuS3W/5alo/Dbgwze8FLCG7F/TOVF9q9ucYxPr4CvAjYEFaLmo9/AA4Jc2PAsYWsS6AdwBP\nAqPT8jyy+5BtUxdu4dd2APDbiFgeEa8D15E9NDZsRcTKiLg/zf8ReBjopPbDckcB10XEGxHxFPBb\nsnpre5I6gY8BV1SsLmI9jAE+FBFXAaTPuIYC1kUyEtha0ihgS+AZ2qguHPBr2wnorlhekdYVgqR3\nAvsCdwHjo/rDcn3r6BmGTx1dCpzNW58TKWI97AI8J+mqlN66XNJWFLAuIuI/gG8AT5N9rjURsYg2\nqgsHfNuApG2A64EzU0u/7539YX2nX9LHgZ70187GugYP63pIRgH7Ad+JiP2Al8nGwyrU7wSApO3I\nWvMTydI7W0s6njaqCwf82p4Bdq5Y7kzrhrX0p+r1wNUR0fuMRI+k8Wn7BOD3af0zZN1rew2XOjoY\nOErSk8C1wF9KuhpYWbB6gOwv2+6IuC8t30D2BVC03wmAw4EnI2JVRLwJ3Ah8kDaqCwf82u4Fdpc0\nUdJo4DNkD40Nd7OBZRHxrYp1vQ/LwVsfllsAfCb1VNgF2B24Z6gKmpeIOC8ido6IXcn+3W+PiM8B\nP6ZA9QCQUhXdkvZIqw4DHqJgvxPJ08BBkraQJLK6WEYb1cVQPGnbliLiTUlfAhaSfTFeGREPN7lY\nuZJ0MHA8sFTSErI/Tc8j64UwX9IU0sNyABGxTNJ8sl/614HTI3VPGKYupJj1cAYwV9JmZL1UTiG7\neVmouoiIeyRdT9bz5vX083JgW9qkLvzglZlZQTilY2ZWEA74ZmYF4YBvZlYQDvhmZgXhgG9mVhAO\n+GZmBeGAbwZI+l9p2NsH0pgx+0s6U9IWzS6b2WBxP3wrPEkHkQ2KdWhEvCFpHLA58Avg/RGxqqkF\nNBskbuGbwduB5yLiDYAU4I8lGyBrsaTbACQdIekXku6TNC+NGomk30m6KL0U4y5Ju6b1n0ovklki\nqdyUT2ZWwS18KzxJWwM/Jxvf/DZgXkTckQZPe39ErJa0PfAvwEcj4k+Svkr2Ioz/Lel3wPcj4kJJ\nnyN7T/MnJD0IfCQinpU0JiJebNJHNAPcwjcjIl4mGwHyNOAPwHWSTkqbe4dHPojsDUZ3pnGGTuSt\no6lel35em/YFuBOYI+lUPG6VtQD/EpoBaVCrO4A7JC0lG/WwkoCFEXF8rVP0nY+IL0jaHzgS+JWk\n/SJi9SAX3axubuFb4UnaQ9LuFav2BZ4CXgLGpHV3AQdL2i0ds5Wkv6g45tPp52eAX6Z9do2IeyNi\nBtkY6ZVjo5sNObfwzbIXtn9b0ljgDbKXTZ8GHAf8TNIzEXGYpFOAayVtTtaK/zuy95QCdEh6AHgV\n+Gxa948VXwqLIuLBIfo8ZlX5pq3ZAKWbtu6+aS3PKR2zgXOrydqCW/hmZgXhFr6ZWUE44JuZFYQD\nvplZQTjgm5kVhAO+mVlBOOCbmRXEfwInsLF+bj4XJAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12a738810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#very simple plotting\n",
    "fig = plt.figure(1)\n",
    "ax1 = fig.add_subplot(111)\n",
    "ax1.set_xlabel('Steps')\n",
    "ax1.set_ylabel('Test Accuracy')\n",
    "ax1.set_title('Accuracy vs Steps')\n",
    "ax1.scatter(step_list, accur_list, s = 3);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "x68f-hxRGm3H"
   },
   "source": [
    "Let's now switch to stochastic gradient descent training instead, which is much faster.\n",
    "\n",
    "The graph will be similar, except that instead of holding all the training data into a constant node, we create a `Placeholder` node which will be fed actual data at every call of `session.run()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "qhPMzWYRGrzM"
   },
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data. For the training data, we use a placeholder that will be fed\n",
    "    # at run time with a training minibatch.\n",
    "    tf_train_dataset = tf.placeholder(tf.float32,\n",
    "                                    shape=(batch_size, image_size * image_size))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "    # Variables.\n",
    "    weights = tf.Variable(tf.truncated_normal([image_size * image_size, num_labels]))\n",
    "    biases = tf.Variable(tf.zeros([num_labels]))\n",
    "  \n",
    "    # Training computation.\n",
    "    logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "  \n",
    "    # Optimizer.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "  \n",
    "    # Predictions for the training, validation, and test data.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(tf_valid_dataset, weights) + biases)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XmVZESmtG4JH"
   },
   "source": [
    "Let's run it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 6
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 66292,
     "status": "ok",
     "timestamp": 1449848003013,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "FoF91pknG_YW",
    "outputId": "d255c80e-954d-4183-ca1c-c7333ce91d0a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 18.240400\n",
      "Minibatch accuracy: 6.2%\n",
      "Validation accuracy: 8.5%\n",
      "Minibatch loss at step 500: 2.371244\n",
      "Minibatch accuracy: 71.1%\n",
      "Validation accuracy: 75.4%\n",
      "Minibatch loss at step 1000: 1.244738\n",
      "Minibatch accuracy: 80.5%\n",
      "Validation accuracy: 76.5%\n",
      "Minibatch loss at step 1500: 1.324030\n",
      "Minibatch accuracy: 75.0%\n",
      "Validation accuracy: 77.2%\n",
      "Minibatch loss at step 2000: 1.123285\n",
      "Minibatch accuracy: 77.3%\n",
      "Validation accuracy: 76.5%\n",
      "Minibatch loss at step 2500: 1.130614\n",
      "Minibatch accuracy: 75.8%\n",
      "Validation accuracy: 78.4%\n",
      "Minibatch loss at step 3000: 0.932454\n",
      "Minibatch accuracy: 77.3%\n",
      "Validation accuracy: 78.8%\n",
      "Minibatch loss at step 3500: 0.729505\n",
      "Minibatch accuracy: 85.9%\n",
      "Validation accuracy: 78.1%\n",
      "Minibatch loss at step 4000: 1.040036\n",
      "Minibatch accuracy: 77.3%\n",
      "Validation accuracy: 79.3%\n",
      "Minibatch loss at step 4500: 0.717898\n",
      "Minibatch accuracy: 82.8%\n",
      "Validation accuracy: 79.5%\n",
      "Minibatch loss at step 5000: 0.574863\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 79.7%\n",
      "Test accuracy: 86.9%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 5001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.initialize_all_variables().run()\n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "        # Pick an offset within the training data, which has been randomized.\n",
    "        # Note: we could use better randomization across epochs.\n",
    "        offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "        # Generate a minibatch.\n",
    "        batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "        batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "        # and the value is the numpy array to feed to it.\n",
    "        feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "        _, l, predictions = session.run(\n",
    "          [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "        if (step % 500 == 0):\n",
    "            print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "            print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "            print(\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n",
    "    print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7omWxtvLLxik"
   },
   "source": [
    "---\n",
    "Problem\n",
    "-------\n",
    "\n",
    "Turn the logistic regression example with SGD into a 1-hidden layer neural network with rectified linear units [nn.relu()](https://www.tensorflow.org/versions/r0.7/api_docs/python/nn.html#relu) and 1024 hidden nodes. This model should improve your validation / test accuracy.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Setting up the graph always comes first\n",
    "\n",
    "batch_size = 128\n",
    "n_hidden_nodes = 1024\n",
    "image_size = 28\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    \n",
    "    # Input data. For the training data, we use a placeholder that will be \n",
    "    # fed at runtime with a training minibatch\n",
    "    tf_train_dataset = tf.placeholder(tf.float32,\n",
    "                                    shape=(batch_size, image_size * image_size))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "    \n",
    "    # Variables.\n",
    "    weights_01 = tf.Variable(\n",
    "    tf.truncated_normal([image_size * image_size, n_hidden_nodes]))\n",
    "    weights_12 = tf.Variable(tf.truncated_normal([n_hidden_nodes, num_labels]))\n",
    "    biases_01 = tf.Variable(tf.zeros([n_hidden_nodes]))\n",
    "    biases_12 = tf.Variable(tf.zeros([num_labels]))\n",
    "\n",
    "    # Training computation.\n",
    "    z_01= tf.matmul(tf_train_dataset, weights_01) + biases_01\n",
    "    h1 = tf.nn.relu(z_01)\n",
    "    z_12 = tf.matmul(h1, weights_12) + biases_12\n",
    "    loss = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(z_12, tf_train_labels))\n",
    "\n",
    "    # Optimizer.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "\n",
    "    # Predictions for the training, validation, and test data.\n",
    "    train_prediction = tf.nn.softmax(z_12)\n",
    "    valid_prediction = tf.nn.softmax(\n",
    "    tf.matmul(tf.nn.relu(tf.matmul(tf_valid_dataset, weights_01) + biases_01), weights_12) + biases_12)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(tf.nn.relu(tf.matmul(tf_test_dataset, weights_01) + biases_01), weights_12) + biases_12) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 324.309784\n",
      "Minibatch accuracy: 10.2%\n",
      "Validation accuracy: 26.4%\n",
      "Minibatch loss at step 500: 15.666269\n",
      "Minibatch accuracy: 75.8%\n",
      "Validation accuracy: 81.2%\n",
      "Minibatch loss at step 1000: 9.701218\n",
      "Minibatch accuracy: 78.9%\n",
      "Validation accuracy: 82.1%\n",
      "Minibatch loss at step 1500: 8.730391\n",
      "Minibatch accuracy: 73.4%\n",
      "Validation accuracy: 81.4%\n",
      "Minibatch loss at step 2000: 5.601308\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 79.5%\n",
      "Minibatch loss at step 2500: 9.332880\n",
      "Minibatch accuracy: 78.1%\n",
      "Validation accuracy: 80.0%\n",
      "Minibatch loss at step 3000: 3.054952\n",
      "Minibatch accuracy: 82.8%\n",
      "Validation accuracy: 80.3%\n",
      "Minibatch loss at step 3500: 3.030112\n",
      "Minibatch accuracy: 88.3%\n",
      "Validation accuracy: 81.5%\n",
      "Minibatch loss at step 4000: 2.456403\n",
      "Minibatch accuracy: 80.5%\n",
      "Validation accuracy: 80.9%\n",
      "Minibatch loss at step 4500: 2.019478\n",
      "Minibatch accuracy: 85.2%\n",
      "Validation accuracy: 83.3%\n",
      "Minibatch loss at step 5000: 0.738836\n",
      "Minibatch accuracy: 89.1%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 5500: 2.041015\n",
      "Minibatch accuracy: 82.8%\n",
      "Validation accuracy: 82.8%\n",
      "Minibatch loss at step 6000: 2.410986\n",
      "Minibatch accuracy: 86.7%\n",
      "Validation accuracy: 83.5%\n",
      "Minibatch loss at step 6500: 2.461916\n",
      "Minibatch accuracy: 81.2%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 7000: 1.873383\n",
      "Minibatch accuracy: 89.8%\n",
      "Validation accuracy: 84.3%\n",
      "Minibatch loss at step 7500: 1.002510\n",
      "Minibatch accuracy: 85.9%\n",
      "Validation accuracy: 83.9%\n",
      "Minibatch loss at step 8000: 0.986571\n",
      "Minibatch accuracy: 91.4%\n",
      "Validation accuracy: 83.3%\n",
      "Minibatch loss at step 8500: 0.415871\n",
      "Minibatch accuracy: 90.6%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 9000: 1.346511\n",
      "Minibatch accuracy: 86.7%\n",
      "Validation accuracy: 84.2%\n",
      "Minibatch loss at step 9500: 0.744818\n",
      "Minibatch accuracy: 89.1%\n",
      "Validation accuracy: 84.5%\n",
      "Minibatch loss at step 10000: 1.356037\n",
      "Minibatch accuracy: 89.1%\n",
      "Validation accuracy: 83.0%\n",
      "Minibatch loss at step 10500: 1.892023\n",
      "Minibatch accuracy: 85.9%\n",
      "Validation accuracy: 84.1%\n",
      "Minibatch loss at step 11000: 0.966964\n",
      "Minibatch accuracy: 88.3%\n",
      "Validation accuracy: 84.0%\n",
      "Minibatch loss at step 11500: 0.858190\n",
      "Minibatch accuracy: 85.9%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 12000: 0.805250\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 12500: 2.565210\n",
      "Minibatch accuracy: 85.2%\n",
      "Validation accuracy: 83.4%\n",
      "Minibatch loss at step 13000: 0.797943\n",
      "Minibatch accuracy: 86.7%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 13500: 0.319276\n",
      "Minibatch accuracy: 91.4%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 14000: 0.922709\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 85.2%\n",
      "Minibatch loss at step 14500: 1.209756\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 84.2%\n",
      "Minibatch loss at step 15000: 0.429418\n",
      "Minibatch accuracy: 89.8%\n",
      "Validation accuracy: 85.3%\n",
      "Minibatch loss at step 15500: 0.444208\n",
      "Minibatch accuracy: 88.3%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 16000: 0.633546\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 84.9%\n",
      "Minibatch loss at step 16500: 0.620418\n",
      "Minibatch accuracy: 91.4%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 17000: 0.509932\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 84.8%\n",
      "Minibatch loss at step 17500: 0.342126\n",
      "Minibatch accuracy: 92.2%\n",
      "Validation accuracy: 85.5%\n",
      "Minibatch loss at step 18000: 0.518216\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 85.7%\n",
      "Minibatch loss at step 18500: 1.201865\n",
      "Minibatch accuracy: 89.8%\n",
      "Validation accuracy: 85.4%\n",
      "Minibatch loss at step 19000: 0.428867\n",
      "Minibatch accuracy: 91.4%\n",
      "Validation accuracy: 85.8%\n",
      "Minibatch loss at step 19500: 0.515053\n",
      "Minibatch accuracy: 93.8%\n",
      "Validation accuracy: 84.7%\n",
      "Minibatch loss at step 20000: 0.336005\n",
      "Minibatch accuracy: 95.3%\n",
      "Validation accuracy: 85.6%\n",
      "Test accuracy: 92.2%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 20001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.initialize_all_variables().run()\n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "        # Pick an offset within the training data, which has been randomized.\n",
    "        # Note: we could use better randomization across epochs.\n",
    "        offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "        # Generate a minibatch.\n",
    "        batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "        batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "        # and the value is the numpy array to feed to it.\n",
    "        feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "        _, l, predictions = session.run(\n",
    "          [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "        if (step % 500 == 0):\n",
    "            cost2_list.append(l)\n",
    "            step2_list.append(step)\n",
    "            print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "            print(\"Minibatch accuracy: %.1f%%\" % accuracy(predictions, batch_labels))\n",
    "            print(\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels))\n",
    "    print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEZCAYAAAC5AHPcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8VXWd//HXG/CgICJQgoLihVJQkTSoSXt4HGdMp0bM\nTFFLy3F0Mh2mfjWB0wynfs6oUzo5UzRNZqGlyC/vXRQNT46WYCqCQooXCEHQQlG8cDuf3x/fdTyb\n4zlwLnvtdfY+7+fjsR577XX9rnX22Z/9+X7X+i5FBGZmZp3Vp+gCmJlZdXIAMTOzLnEAMTOzLnEA\nMTOzLnEAMTOzLnEAMTOzLnEAMTOzLnEAscJJapS0TtJORZelJ5M0UtJPJb0k6WVJiySdlc0bLalJ\nkv+nrWL8YbNCSRoNHAU0ASdWeN99K7m/MrgOWAHsDQwDPg2szeYJiOzVrCIcQKxoZwG/BX4EfKZ0\nhqSdJV0haXn2i/s+Sf2zeUdJeiCbvqLkl/i9ks4p2cbZkv635H2TpAskPQU8lU37lqQ/SFov6SFJ\nR5Us30fSxZKelvRqNn+kpG9L+mar8t4maWrrA5Q0U9I3Wk27VdI/ZONfkfR8tv2lko5p51xNBGZF\nxFsR0RQRj0XEXdm8X2evr2Tb+UC27XMkLZH0J0m/lLRPq3NxkaRnJL0o6d9L5h2QZYavZPNuaKdM\n1ptFhAcPhQ3AMuB84HBgE/DuknnfAeYBI0i/rD8I7ATsA7wKnAr0BYYA47N17gXOKdnG2cB9Je+b\ngLuAwUD/bNoZwO6kH1RfAF4A6rJ5XwYeA8Zk7w/N9jcReL5ku8OADcC72jjGDwMrSt7vDrwODAfe\nC/wBGJ7N2wfYr51zNRe4HzgN2LvVvNHAVkAl0yaTguR7s2O7GHig1bn4VXYuRgFPNp874HpgejZe\nB3yo6M+Kh543FF4AD713IFVdbQSGZO+XAFOzcQFvAIe0sd404KZ2ttmRAHL0Dsq1Djg0G/898LF2\nlnsCODYb/zzws+1sczlwVDZ+LnBPNn4AsAY4Fui3g3INBv4NWAxsBh4B3p/Naw4gfUqW/wXw2ZL3\nfbLAtXfJufjLkvmfA+7OxmcB/w2MLPpz4qHnDq7CsiKdBcyNiJez9zeQvvAB3gX0B55tY729gWe6\nsd/nS99I+lJWzfOypJeB3bL9N++rrTIAXAt8Khv/FKmNoj03Aqdn42cAPwGIiGeAfwAagLWSrpe0\nZ1sbiIj1EXFxRBxKyl4eA27Zzj5HA1dlFyisA/5EaicZWbJM6blYAeyVjf8jKeAskLRY0me3sx/r\npRxArBCSdiZVQR0t6QVJL5C+SA+TdCjwR+At0i/01lYCY9rZ9OvAgJL3I9pY5u0uqLP2ji8Dp0TE\nkIgYQqoea26MXtlOGQB+DEyWNB44CLi1neUgBcdTsjaIDwA3vV2YiNkR8WHSFz7AZdvZTvM664Bv\nAntJGlJ6TCX+AJwfEUOzYUhE7BoRD5Yss3fJ+D7A6mz7ayPivIgYCfwdMFPS/jsql/UuDiBWlI8D\nW4CxwGHZMJZUx39WRATwQ+BKSXtmjdkfzC71/QlwrKRTJPWVNFTSYdl2FwInS9pF0hjgb3ZQjkGk\n6qA/SaqT9C/ZtGZXA/832xaSDs2+sImIVcDvSJnHTRGxsb2dRMRCUgZwNXBnRLyabe+9ko6RVEdq\nA3qTVLX0DpIuk3RwdsyDgAuAp7MM7qVsvdJg9z3gYknjsvUHSzql1Wa/LGl3SXsDU4HZ2bKnSGrO\nVF7Jtt1muaz3cgCxopwFXBMRqyLixeYB+DZwZnY/w5dI9f0Pkb58LyPV8a8E/iqbvw54FBifbfc/\nSAFhDSkA/bjVflv/Ur8rG54CniO1u6wsmX8lMAeYK2k9KQDsUjJ/FnAIqTprR64ntXX8pGRa/+y4\nXiL9+n83ML2d9QeQqqxeBp4mZQ8nAkTEm8C/Ag9kVVaTIuLWbNuzJb0CLAKOb7XN24CHSe0pdwDX\nZNMnAvMlvUrKrP4+IpZ34BitF1H6oZfzTtKXwe9IV62cmP2Cu5GUsi8HTo2I9dmy04FzSL9Op0bE\n3NwLaNZFkj4MXBcR+xZdls6S1ES6uqy9Nh6z7apUBjKVdIVNs2mkq1AOJF2mOR0gS7VPJVVlnECq\nd/WNUdYjZdVpU4HvF10WsyLkHkAkjSJVN1xdMnkyKfUnez0pGz8RmB0RW7J0eRkwKe8ymnWWpINI\nVUnDgasKLk5X+XnW1i39KrCP/yBd5TK4ZNrwiFgLEBFrJO2RTR9Juiu52Sq2veTQrEeIiN8DuxZd\nju6IiGrrysV6mFwzEEkfBdZmV6BsryrKv4TMzKpM3hnIkcCJkv6KdOXKIEnXAWskDY+ItZJGAC9m\ny69i2+vSR2XTtiHJAcfMrAsiomztyrlmINlds/tExP7AFGBeRHyadLngZ7LFziZdSghwOzAlux5/\nP9LNYgva2baHCGbMmFF4GXrK4HPhc+Fzsf2h3CrRBtKWy4A5Wa+pK0hXXhERSyTNIV2xtRm4IPI4\najMz67aKBZCI+DVZl9ORumH4i3aWuxS4tFLlMjOzrvGd6FWuvr6+6CL0GD4XLXwuWvhc5Kcid6KX\nmyTXbJmZdZIkoloa0c3MrHZVbQDZurXoEpiZ9W5VG0DWry+6BGZmvVvVBpB164ougZlZ71a1AeTl\nl3e8jJmZ5adqA4gzEDOzYlVtAHEGYmZWLAcQMzPrkqoNIK7CMjMrVtUGEGcgZmbFqtoA4gzEzKxY\nVRtAnIGYmRWragOIMxAzs2JVbQBxBmJmVqyqDSDOQMzMilW1AcQZiJlZsao2gGzeDBs3Fl0KM7Pe\nK9cAIqm/pPmSHpW0WNKMbPoMSc9LeiQbji9ZZ7qkZZKWSjquvW0PGeIsxMysSP3y3HhEbJR0TES8\nIakv8ICkX2azr4yIK0uXlzQWOBUYC4wC7pH0nraeXzt0aAogI0bkeQRmZtae3KuwIuKNbLQ/KWA1\nB4O2nss7GZgdEVsiYjmwDJjU1naHDHFDuplZkXIPIJL6SHoUWAPcHREPZbMulLRQ0tWSBmfTRgIr\nS1ZflU17h+YMxMzMilGJDKQpIt5HqpKaJGkcMBPYPyImkALLFZ3drjMQM7Ni5doGUioiXpXUCBzf\nqu3j+8Ad2fgqYO+SeaOyae/w1FMNvPQSPPss1NfXU19fn0OpzcyqV2NjI42NjbltX220T5dv49K7\ngM0RsV7SLsBdwGXAIxGxJlvmC8DEiDgjy05+AnyAVHV1N/CORnRJMWNGEAFf+1puxTczqymSiIi2\n2p+7JO8MZE9glqQ+pOqyGyPiF5KulTQBaAKWA+cDRMQSSXOAJcBm4IK2rsCCVIX1zDM5l97MzNqV\nawaSF0lx7bXBXXfBj39cdGnMzKpDuTOQqr0T3TcSmpkVq2oDyNChvgrLzKxIVRtAnIGYmRWragOI\nMxAzs2JVbQAZMgReeQWq8BoAM7OaULUBpK4uDa+/XnRJzMx6p6oNIODuTMzMilT1AcQN6WZmxajq\nAOKGdDOz4lR1AHEGYmZWnKoOIM5AzMyKU9UBxBmImVlxqjqAOAMxMytOVQcQZyBmZsWp6gDiDMTM\nrDhVHUCcgZiZFaeqA4gzEDOz4lR1AHEGYmZWnKoOIEOHOoCYmRUl1wAiqb+k+ZIelbRY0oxs+hBJ\ncyU9KekuSYNL1pkuaZmkpZKO2972Bw+G116DrVvzPAozM2tLrgEkIjYCx0TE+4AJwAmSJgHTgHsi\n4kBgHjAdQNI44FRgLHACMFNSuw+A79MHBg2C9evzPAozM2tL7lVYEfFGNtof6AcEMBmYlU2fBZyU\njZ8IzI6ILRGxHFgGTNre9t2QbmZWjNwDiKQ+kh4F1gB3R8RDwPCIWAsQEWuAPbLFRwIrS1ZflU1r\nlxvSzcyK0S/vHUREE/A+SbsBt0g6mJSFbLNYZ7fb0NAApOBx7731TJxY382SmpnVlsbGRhobG3Pb\nvqKCDxWX9M/AG8C5QH1ErJU0Arg3IsZKmgZERFyeLX8nMCMi5rfaTjSX+7TT4OMfhylTKnYYZmZV\nSRIR0W67cmflfRXWu5qvsJK0C/CXwFLgduAz2WJnA7dl47cDUyTVSdoPGAMs2N4+3AZiZlaMvKuw\n9gRmSepDClY3RsQvJD0IzJF0DrCCdOUVEbFE0hxgCbAZuCB2kCK5DcTMrBi5BpCIWAwc3sb0dcBf\ntLPOpcClHd3H0KHwwgtdLqKZmXVRVd+JDs5AzMyKUvUBxN2ZmJkVo+oDyJAhbkQ3MytC1QcQZyBm\nZsWo+gDiDMTMrBg1EUCcgZiZVV7VB5CBA2HzZti4seiSmJn1LlUfQCRnIWZmRaj6AALuzsTMrAg1\nEUCcgZiZVV5NBBBnIGZmlVcTAcQZiJlZ5dVEAHEGYmZWeTURQJyBmJlVXk0EEHdnYmZWeTURQNyd\niZlZ5dVMAHEGYmZWWTURQNyIbmZWeTURQJyBmJlVXq4BRNIoSfMkPSFpsaSLsukzJD0v6ZFsOL5k\nnemSlklaKum4juzHGYiZWeUpIvLbuDQCGBERCyXtCjwMTAZOA16LiCtbLT8WuB6YCIwC7gHeE60K\nKWmbSZs2pV55N21KnSuamdk7SSIiyvYtmWsGEhFrImJhNr4BWAqMzGa3dRCTgdkRsSUilgPLgEk7\n2k9dHfTvDxs2lKfcZma2YxVrA5G0LzABmJ9NulDSQklXSxqcTRsJrCxZbRUtAWe73A5iZlZZ/Sqx\nk6z66qfA1IjYIGkm8PWICEmXAFcA53Zmmw0NDW+P19fXM3RoPevWwT77lLHgZmZVrLGxkcbGxty2\nn2sbCICkfsDPgF9GxFVtzB8N3BER4yVNAyIiLs/m3QnMiIj5rdZp3SxCfT3MmAHHHJPTgZiZVbmq\nagPJXAMsKQ0eWeN6s5OBx7Px24Epkuok7QeMARZ0ZCe+EsvMrLJyrcKSdCRwJrBY0qNAABcDZ0ia\nADQBy4HzASJiiaQ5wBJgM3DBO1KNdrgNxMyssnINIBHxANC3jVl3bmedS4FLO7svd6hoZlZZNXEn\nOrhDRTOzSqupAOIMxMyscmomgLgR3cyssmomgDgDMTOrrJoJIM5AzMwqq2YCiDMQM7PKqpkA4gzE\nzKyycu/KJA9tdWXS1JR65d24Efq2deeJmVkvV41dmVREnz6w227wyitFl8TMrHeomQACbgcxM6uk\nmgog7s7EzKxyaiqAuDsTM7PKqakA4gzEzKxyaiqAOAMxM6ucmgsgzkDMzCqjQwFE0nUdmVY030xo\nZlY5Hc1ADi59I6kvcET5i9M9zkDMzCpnuwFE0nRJrwHjJb2aDa8BLwK3VaSEneAMxMyscrYbQCLi\n0ogYBHwjInbLhkERMSwipleojB3mDMTMrHI6WoX1M0kDASR9StKVkkbvaCVJoyTNk/SEpMWS/j6b\nPkTSXElPSrpL0uCSdaZLWiZpqaTjOnMwzkDMzCqnowHku8Abkg4D/g/wDHBtB9bbAnwxIg4G/gz4\nvKSDgGnAPRFxIDAPmA4gaRxwKjAWOAGYKanDHX85AzEzq5yOBpAtWfe3k4FvR8R3gEE7Wiki1kTE\nwmx8A7AUGJVtZ1a22CzgpGz8RGB2RGyJiOXAMmBSB8voDMTMrII6GkBekzQd+DTwc0l9gJ06syNJ\n+wITgAeB4RGxFlKQAfbIFhsJrCxZbVU2rUMGDIAtW1KX7mZmlq9+HVzuNOAM4JyIWCNpH+AbHd2J\npF2BnwJTI2KDpNYPIen0Q0kaGhreHq+vr6e+vh6ppTuTESM6u0Uzs9rS2NhIY2Njbtvv8AOlJA0H\nJmZvF0TEix1crx/wM+CXEXFVNm0pUB8RayWNAO6NiLGSpgEREZdny90JzIiI+a22+Y4HSjUbOxZu\nugnGjevQYZmZ9RqFPFBK0qnAAuCTpEbu+ZJO6eA+rgGWNAePzO3AZ7Lxs2m5p+R2YIqkOkn7AWOy\n/XaYG9LNzCqjo1VY/wRMbM46JL0buIdULdUuSUcCZwKLJT1Kqqq6GLgcmCPpHGAFKSgREUskzQGW\nAJuBC9pNNdrhhnQzs8roaADp06rK6k90IHuJiAeA9p5Q/hftrHMpcGkHy/UOzkDMzCqjowHkTkl3\nATdk708DfpFPkbrHGYiZWWVsN4BIGkO65PbLkk4Gjspm/Rb4Sd6F6wpnIGZmlbGjaqhvAa8CRMTN\nEfHFiPgicEs2r8dxBmJmVhk7CiDDI2Jx64nZtH1zKVE3OQMxM6uMHQWQ3bczb5dyFqRcnIGYmVXG\njgLI7yT9beuJks4FHs6nSN3jDMTMrDJ2dBXWPwC3SDqTloDxfqAO+HieBesqZyBmZpWx3QCSdXj4\nIUnHAIdkk38eEfNyL1kXOQMxM6uMDveF1ZNsry+sTZtg4MD02vEniZiZ1b5C+sKqJnV10L8/bNhQ\ndEnMzGpbzQUQcDWWmVkl1GQAcUO6mVn+ajKAOAMxM8tfTQYQZyBmZvmryQDiDMTMLH81GUCcgZiZ\n5a8mA4gzEDOz/NVkAHEGYmaWv5oMIM5AzMzyl2sAkfQDSWslLSqZNkPS85IeyYbjS+ZNl7RM0lJJ\nx3V1v85AzMzyl3cG8kPgI21MvzIiDs+GOwEkjQVOBcYCJwAzpa71ZuUMxMwsf7kGkIi4H2jrq7yt\nwDAZmB0RWyJiObAMmNSV/Q4d6gBiZpa3otpALpS0UNLVkgZn00YCK0uWWZVN67QhQ1yFZWaWtx09\nUCoPM4GvR0RIugS4Aji3sxtpaGh4e7y+vp76+vq33w8enHrj3boV+vbtdnnNzKpSY2MjjY2NuW0/\n9+eBSBoN3BER47c3T9I0ICLi8mzencCMiJjfxnrtPg+k2dChsGwZDBtWlsMwM6t61fg8EFHS5iFp\nRMm8k4HHs/HbgSmS6iTtB4wBFnR1p25INzPLV65VWJKuB+qBYZL+AMwAjpE0AWgClgPnA0TEEklz\ngCXAZuCCHaYZ2+FLec3M8pVrAImIM9qY/MPtLH8pcGk59u0MxMwsXzV5Jzo4AzEzy1vNBhBnIGZm\n+arZAOIMxMwsXzUbQJyBmJnlq2YDiLszMTPLV80GEHdnYmaWr5oOIM5AzMzyU7MBxI3oZmb5qtkA\n4gzEzCxfNRtAnIGYmeWrZgPIgAGwZQu89VbRJTEzq001G0AkX8prZpanmg0g4HYQM7M81XQAcTuI\nmVl+ajqAOAMxM8tPTQcQZyBmZvmp6QDiDMTMLD81HUB8FZaZWX5qOoC4Q0Uzs/zkGkAk/UDSWkmL\nSqYNkTRX0pOS7pI0uGTedEnLJC2VdFx39+8qLDOz/OSdgfwQ+EiradOAeyLiQGAeMB1A0jjgVGAs\ncAIwU5K6s3M3opuZ5SfXABIR9wOtc4DJwKxsfBZwUjZ+IjA7IrZExHJgGTCpO/t3BmJmlp8i2kD2\niIi1ABGxBtgjmz4SWFmy3KpsWpc5AzEzy0+/ogsARFdWamhoeHu8vr6e+vr6dyzjDMTMerPGxkYa\nGxtz274iuvT93fEdSKOBOyJifPZ+KVAfEWsljQDujYixkqYBERGXZ8vdCcyIiPltbDM6Uu5Nm2Dg\nwPTavdYUM7PqJ4mIKNu3YSWqsJQNzW4HPpONnw3cVjJ9iqQ6SfsBY4AF3dlxXR307w8bNnRnK2Zm\n1pZcq7AkXQ/UA8Mk/QGYAVwG/D9J5wArSFdeERFLJM0BlgCbgQs6lGbsQHM7yKBB3d2SmZmVyr0K\nKw8drcICOOwwmDULJkzIuVBmZj1cNVZhFcpXYpmZ5aPmA4ivxDIzy0fNBxB3qGhmlo+aDyDuUNHM\nLB+9IoA4AzEzK7+aDyBuRDczy0fNBxBnIGZm+aj5AOIMxMwsHzUfQJyBmJnlo+YDiDMQM7N81HwA\ncQZiZpaPmu8Lq6kp9cq7cSP07ZtzwczMejD3hdVJffrAbrvBK68UXRIzs9pS8wEE3J2JmVkeekUA\ncXcmZmbl12sCiDMQM7Py6hUBxJfympmVX68IIM5AzMzKL9dnom+PpOXAeqAJ2BwRkyQNAW4ERgPL\ngVMjYn139+UMxMys/IrMQJqA+oh4X0RMyqZNA+6JiAOBecD0cuzIGYiZWfkVGUDUxv4nA7Oy8VnA\nSeXYkTMQM7PyKzKABHC3pIcknZtNGx4RawEiYg2wRzl25AzEzKz8CmsDAY6MiBckvRuYK+lJUlAp\nVZZ+VpyBmJmVX2EBJCJeyF5fknQrMAlYK2l4RKyVNAJ4sb31Gxoa3h6vr6+nvr6+3X05AzGz3qix\nsZHGxsbctl9IZ4qSBgB9ImKDpIHAXOBrwLHAuoi4XNJXgCERMa2N9TvcmSLA88/DBz4Aq1aV6QDM\nzKpQuTtTLCoDGQ7cIimyMvwkIuZK+h0wR9I5wArg1HLszBmImVn51Xx37gARsPPOsH59ejUz643c\nnXsXSM5CzMzKrVcEEHCPvGZm5dZrAsjRR8NZZ8ETTxRdEjOz2tBrAsh3vwuf+xzU18M3vwlbtxZd\nIjOz6tYrGtFLPfccfPazsGUL/OhHMGZMectmZtZTuRG9m/bbD+bNg09+Ev7sz1JmUq4Y+tZbcN11\ncOaZsHBhebZpZtZT9boAAtCnD0ydCv/7vykL+chHYOXKrm9vxQqYPh1Gj4Yf/xgOPhiOOw7+9V9T\npmNmVot6ZQBpdtBB8MADqV3kiCNg1qyOZyMRcPfdcNJJcPjh8OabKSDddRdcfDE8/DD8+tdw5JHw\n+9/nehhmZoXodW0g7XnssXSV1n77wfe+B8OHt73c+vUp0MycCf37w+c/n6qsBg5857IRqYrsX/4F\n/vmf4aKLUvZjZlYEt4Hk5LDDYMGCVP102GHw059uO3/xYvi7v4N994Xf/Aa+//3UznHeeW0HD0g3\nMF5wATz4IMyZA8ceC8uX530kZmaV4QykDfPnp2zkiCPgr/8a/vu/4emn4fzz4W//Fvbcs/Pb3LoV\nrrwS/v3f4dJL4W/+JgUYM7NKKXcG4gDSjjfegK9+NWUe552X2jp22qn723388RSc9torZTFdCUZW\nfV58EWbPhk99Kj2fxqwIDiBUJoDkadMmuOSS1NZy1VUwZUrH141IX0bPPJOyoqYmOO002GWX/Mpr\n3XPffXDGGal69LHH0t/+nHPcHmaV5wBC9QeQZg89lLKR8ePhO9+Bd70rTW9qSs8uefrplkBROt6/\nf7oB8oADUqP+ww/DF7+Y2mgGDSr2mKxFUxNcfnn6kfCjH8Hxx8Mjj6QLL5qa0t/8/e8vupTWmziA\nUDsBBNLlv1/9aqreOOKIFCCeey5VcxxwQEugGDOmZXz33bfdxqJF8G//Br/6VbrS66KLUueRVpw/\n/hE+/Wl47bX0tx01qmVeU1O6ku/ii2Hy5HS/0LBhxZXVeg8HEGorgDRbsABWr04BYv/927+ya3ue\negouuwxuuy2123zhC7DHHuUvq23fAw/A6aen4ZJL2m87e+WVdIn3jTfC178O554LfftWtqxWPk89\nBbvumto3eyoHEGozgJTTihXpaq8bbkhVZF/60ra/gHuq11+H+++Hxkbo1y+1GYwbB+99b3U8CKyp\nKXXUecUV8IMfwMc+1rH1Fi6ECy+EjRvh299Oj1+2ni8i/e1uugluvjlVJ7/5ZqqqnDq1Z/4dHUBw\nAOmo1avTl9kPfwinnALTpqXspqfYtCldMj1vXqp+e+SRdFf/Mcek+U88AUuWwLPPpm5ixo1rCSrj\nxqWeBHpKYPnTn+Dss9MzZ2bPhn326dz6Eakfta98BT760XSp97vfnU9ZreuamtJ9XTffnIY+feAT\nn4CTT4aJE+HVV+Gaa+C//gtGjEiB5BOfKM8VnOXgAIIDSGf98Y+pIfe734UTTkiBZPjw9KXXPKxb\nt+37toZ+/VraYkrbZMaMSWn7jq4q2ro1/WJrDhi/+U3KLv78z9NNlkcd1XbV3aZNsGxZCiZLlrQE\nlmeeSZlVc2BprvobMCANu+zS/ng5r4B68MF0Jd0pp6Qv/u58WaxfDzNmwPXXQ0NDuveodbXWm2/C\nCy+kCy1Wr05D6fjq1akPtvHj4X3vgwkT0rDPPr73qCu2bEndEt18M9xyS2qfbA4a48e3fU63boU7\n7oBvfSu1a15wQapWbr5QppI2bUrVa48/Dqef3gsCiKTjgW+R7pT/QURc3mq+A0gXrF+frvz5z/9M\nH6phwzo3bNyYvrRLrwxrHtavT1/grRv9hw2D3/42BYzGxvSrrDlgHH109+6J2Lw57bs5qKxYke7f\neeON9CXbPN76/ZtvQl1dCiYDB8J73pN6Hxg/Pr2OG9exzCYifUFcdhn8z/+kBvFyWbw4Xa21YUMq\nU2mgeP31dP/QyJEpcO+117bje+2VvtQWLUoBe+FCePTRdNzNwaQ5sIwd23N+HXdFRPpcvvVW+69b\nt6a/9047tf3aelrfvmndu+9OQeP221MXR5/4BHz843DggZ0r48KF6QfcrbemHxlTp8Ihh5T/XDQ1\npf+BxYtTsGh+ffrplMEfcgjcdFONBxBJfYCngGOB1cBDwJSI+H3JMg4gmcbGRurr64suBhs2pKqm\n1oHlxRdh0qQUMI45Jt8Gxo6ei+YvnTfeSFUOTz6Z7s9YtCi9Pv10CoalQeWww9KXdvOvzZdfTs+V\nWb06dVOz777lP56I9KWzbt22QWLYsB1nEm2di7Vr0/GVBpUVK1JV4IQJ6RhHj24JQsOHly+4tM6a\n1q5N0956q+W19Xhb75uH5uCwaVP64t9553R5+847bzvevz+89lojAwfWs2lT+tGxaRPbjJdO27Qp\nZad9+6Y2jOagMXp098/Biy+me79mzkwZ89SpqbqyM9nw1q3pf+3VV1NWvnhxS6B44ol0heYhh8Ch\nh7a8HnRQy31iNV+FJemDwIyIOCF7Pw2I0izEAaRFQ0MDDQ0NRRejRyjXudi4EZYubQkoza9NTelL\n9tBD06/SE09MFyvU1XW/7OXW0XPx+uvpy2fhwnScK1e2ZDsvvZSCVWlm09YQ0RIY2nvdsGHbrGn4\n8JbqxOYv/dLx1u932SUFg+bX5uBQV7fjL+DOfC4i0pd0U1N+f9dNm9KPjquuSj9Ezjgj7fe119Kw\nYUPLeOspbGGXAAAGGElEQVT3GzemrHnQoJTplwaKgw/e8eX75Q4g/cq1oTIaCZQ+neN5YFJBZbFe\nqH//lqqeZhEtv+Afeyw1kn70o8WVsVwGDky/tNu6YmjLlvSrefXqlD00B5aHHtq2Sk1qCQzNrx/6\n0Duzpmq4815KbX15qqtLXdqceWZqB/z5z1NgHD06BYZBg9LlwG2NDxjQs9qxemIAMetxpNR+M2JE\negBZb9CvX0uWYeUnpecFHXlk0SXpup5ahdUQEcdn79uswiqqfGZm1azW20D6Ak+SGtFfABYAp0fE\n0kILZmZm2+hxVVgRsVXShcBcWi7jdfAwM+thelwGYmZm1aEKrovYlqTjJf1e0lOSvlJ0eSpB0nJJ\nj0l6VNKCbNoQSXMlPSnpLkmDS5afLmmZpKWSjiuu5N0n6QeS1kpaVDKt08cu6XBJi7LPzbcqfRzl\n0M65mCHpeUmPZMPxJfNq8lxIGiVpnqQnJC2W9PfZ9F73uWjjXFyUTa/M5yIiqmYgBbyngdHATsBC\n4KCiy1WB434WGNJq2uXAP2bjXwEuy8bHAY+Sqif3zc6Xij6Gbhz7UcAEYFF3jh2YD0zMxn8BfKTo\nYyvTuZgBfLGNZcfW6rkARgATsvFdSW2mB/XGz8V2zkVFPhfVloFMApZFxIqI2AzMBsrYgUSPJd6Z\nLU4GZmXjs4CTsvETgdkRsSUilgPLqOL7aCLifuDlVpM7deySRgCDIuKhbLlrS9apGu2cC0ifj9Ym\nU6PnIiLWRMTCbHwDsBQYRS/8XLRzLkZms3P/XFRbAGnrJsOR7SxbSwK4W9JDks7Npg2PiLWQPkRA\n85M/Wp+jVdTeOdqjk8c+kvRZaVZrn5sLJS2UdHVJtU2vOBeS9iVlZQ/S+f+JWj0X87NJuX8uqi2A\n9FZHRsThwF8Bn5f0YVJQKdWbr4bozcc+E9g/IiYAa4ArCi5PxUjaFfgpMDX79d1r/yfaOBcV+VxU\nWwBZBZQ+aWFUNq2mRcQL2etLwK2kKqm1koYDZOnni9niq4C9S1avxXPU2WOv2XMSES9FVmkNfJ+W\n6sqaPheS+pG+MK+LiNuyyb3yc9HWuajU56LaAshDwBhJoyXVAVOA2wsuU64kDch+XSBpIHAcsJh0\n3J/JFjsbaP4nuh2YIqlO0n7AGNLNmNVMbFuf26ljz6oz1kuaJEnAWSXrVJttzkX2RdnsZODxbLzW\nz8U1wJKIuKpkWm/9XLzjXFTsc1H0VQRduOrgeNKVBsuAaUWXpwLHux/parNHSYFjWjZ9KHBPdi7m\nAruXrDOddHXFUuC4oo+hm8d/Palb/43AH4DPAkM6e+zAEdn5WwZcVfRxlfFcXAssyj4jt5LaAWr6\nXABHAltL/i8eyb4XOv0/UcPnoiKfC99IaGZmXVJtVVhmZtZDOICYmVmXOICYmVmXOICYmVmXOICY\nmVmXOICYmVmXOICYbYekf5L0uFJ3+o9ImihpqqSdiy6bWdF8H4hZOyR9kNSH0NERsUXSUKA/8Bvg\niIhYV2gBzQrmDMSsfXsCf4yILQBZwDgF2Au4V9KvACQdJ+k3kn4n6UZJA7Lpz0m6PHtIz4OS9s+m\nfzJ7+M+jkhoLOTKzMnAGYtaOrO+x+4FdgF8BN0bEfZKeJWUgL0saBtwMHB8Rb0r6R6AuIi6R9Bzw\nvYi4TNKngVMj4q+Vnij4kYh4QdJuEfFqQYdo1i3OQMzaERGvA4cD5wEvAbMlnZ3Nbu7Q8IOkJ949\nIOlRUid0pT1Gz85eb8iWBXgAmJU926Vffkdgli9/eM22I1KKfh9wn6TFpF5eSwmYGxFntreJ1uMR\n8TlJE4GPAQ9LOjwi2nrSoFmP5gzErB2S3itpTMmkCcBy4DVgt2zag8CRkg7I1hkg6T0l65yWvU4B\nfpsts39EPBQRM0jPrCh9DoNZ1XAGYta+XYH/yh4HuoXUBfZ5wBnAnZJWRcSxkj4L3CCpPynL+Cqp\nS2yAIZIeA94CTs+mfaMkyNwTEYsqdDxmZeVGdLOcZI3ovtzXaparsMzy419nVtOcgZiZWZc4AzEz\nsy5xADEzsy5xADEzsy5xADEzsy5xADEzsy5xADEzsy75//3vToW7kOlPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12a71bdd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#very simple plotting\n",
    "fig = plt.figure(1)\n",
    "ax1 = fig.add_subplot(111)\n",
    "ax1.set_xlabel('Steps')\n",
    "ax1.set_ylabel('Cost')\n",
    "ax1.set_title('Accuracy vs Steps')\n",
    "ax1.plot(step2_list,cost2_list,);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "2_fullyconnected.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
